# -*- coding: utf-8 -*-
"""
运行环境:win10+scrapy1.6.0+python3.7.3+pymysql
运行方式:进入MapSpider目录(scrapy.cfg目录)输入：
        scrapy crawl baidu
"""
import scrapy
import json
from scrapy.loader import ItemLoader
from MapSpider.items import MapSqlItem
from fake_useragent import UserAgent

class BaiduSpider(scrapy.Spider):
    name = 'baidu'
    page = 1
    heads = {}
    url   = ''
    
    #start quote
    def start_requests(self):
        ak = 'L113eWb7ti2bUIpAsFxYUByQ'
        self.url = 'http://api.map.baidu.com/place/v2/search?query=%s&tag=%s&region=%s&output=%s&ak=%s'
        self.url = self.url%(self.settings.get('QUOTE_QUERY_KEYWORD'), self.settings.get('QUOTE_QUERY_TAG'), self.settings.get('QUOTE_QUERY_REGION'),'json',ak)

        ua = UserAgent()
        self.headers = {'User-Agent':ua.random, 'Referer':'https://map.baidu.com/@11587831,3559917,13z'}

        requestUrl = self.url+'&page_num='+str(self.page)
        self.page += 1

        yield scrapy.Request(url=requestUrl, callback=self.parse, method='GET', headers=self.headers)

    #parse the data 
    def parse(self, response):

        responseJson = json.loads(response.text)
        result = responseJson.get('results')

        if result:
            im = ItemLoader(item=MapSqlItem())
            for quote in result:
                mobile = quote.get('telephone')
                if not mobile:
                    continue

                im.add_value('name', quote.get('name'))
                im.add_value('mobile', mobile)
                im.add_value('provice', quote.get('province'))
                im.add_value('city', quote.get('city'))
                im.add_value('area', quote.get('area'))
                im.add_value('detail',json.dumps(quote))

            yield im.load_item()

            #loop start_requests
            nextUrl = self.url+'&page_num='+str(self.page)
            self.page += 1 
            yield scrapy.Request(url=nextUrl, callback=self.parse, method='GET', headers=self.headers)

        else:
            return []
        